{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "41eae4c3",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b23863b8",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "# 任务1：报名比赛"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "ee9f3fd9",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "train_cn = pd.read_excel('data/中文_trian.xlsx')\n",
    "train_ja = pd.read_excel('data/日语_train.xlsx')\n",
    "train_en = pd.read_excel('data/英文_train.xlsx')\n",
    "test_ja = pd.read_excel('data/testA.xlsx', sheet_name='日语_testA')\n",
    "test_en = pd.read_excel('data/testA.xlsx', sheet_name='英文_testA')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "7f61c65f",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>原始文本</th>\n",
       "      <th>意图</th>\n",
       "      <th>槽值1</th>\n",
       "      <th>槽值2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>16.5度</td>\n",
       "      <td>adjust_ac_temperature_to_number</td>\n",
       "      <td>offset:16.5</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>16度</td>\n",
       "      <td>adjust_ac_temperature_to_number</td>\n",
       "      <td>offset:16</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>16空调开到16度</td>\n",
       "      <td>adjust_ac_temperature_to_number</td>\n",
       "      <td>offset:16</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>16温度16度</td>\n",
       "      <td>adjust_ac_temperature_to_number</td>\n",
       "      <td>offset:16</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>17度</td>\n",
       "      <td>adjust_ac_temperature_to_number</td>\n",
       "      <td>offset:17</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32270</th>\n",
       "      <td>转换到2D视图</td>\n",
       "      <td>view_trans</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32271</th>\n",
       "      <td>转换到地图模式</td>\n",
       "      <td>view_trans</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32272</th>\n",
       "      <td>转换一下视觉啊</td>\n",
       "      <td>view_trans</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32273</th>\n",
       "      <td>转为专业导航</td>\n",
       "      <td>view_trans</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32274</th>\n",
       "      <td>转为专业导航模式</td>\n",
       "      <td>view_trans</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>32275 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            原始文本                               意图          槽值1  槽值2\n",
       "0          16.5度  adjust_ac_temperature_to_number  offset:16.5  NaN\n",
       "1            16度  adjust_ac_temperature_to_number    offset:16  NaN\n",
       "2      16空调开到16度  adjust_ac_temperature_to_number    offset:16  NaN\n",
       "3        16温度16度  adjust_ac_temperature_to_number    offset:16  NaN\n",
       "4            17度  adjust_ac_temperature_to_number    offset:17  NaN\n",
       "...          ...                              ...          ...  ...\n",
       "32270    转换到2D视图                       view_trans          NaN  NaN\n",
       "32271    转换到地图模式                       view_trans          NaN  NaN\n",
       "32272    转换一下视觉啊                       view_trans          NaN  NaN\n",
       "32273     转为专业导航                       view_trans          NaN  NaN\n",
       "32274   转为专业导航模式                       view_trans          NaN  NaN\n",
       "\n",
       "[32275 rows x 4 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_cn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "9216941b",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>原始文本</th>\n",
       "      <th>中文翻译</th>\n",
       "      <th>意图</th>\n",
       "      <th>槽值1</th>\n",
       "      <th>槽值2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>エアコンのスイッチONに</td>\n",
       "      <td>打开空调开关</td>\n",
       "      <td>open_ac</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>エアコン入れる</td>\n",
       "      <td>打开空调</td>\n",
       "      <td>open_ac</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>エアコンのスイッチを</td>\n",
       "      <td>打开空调开关</td>\n",
       "      <td>open_ac</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>エアコンのスイッチ入れる</td>\n",
       "      <td>打开空调开关</td>\n",
       "      <td>open_ac</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>エアコンのスイッチON</td>\n",
       "      <td>打开空调开关</td>\n",
       "      <td>open_ac</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>997</th>\n",
       "      <td>気に入った楽曲リストをかける</td>\n",
       "      <td>播放喜欢的歌曲列表</td>\n",
       "      <td>play_collect_music</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>998</th>\n",
       "      <td>気に入った歌一覧を再生する</td>\n",
       "      <td>播放喜欢的歌曲</td>\n",
       "      <td>play_collect_music</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>999</th>\n",
       "      <td>お気に入りのミュージックを聴きたいんですけど</td>\n",
       "      <td>我想听喜欢的音乐</td>\n",
       "      <td>play_collect_music</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1000</th>\n",
       "      <td>音楽のお気に入り一覧を再生する</td>\n",
       "      <td>播放喜欢的音乐</td>\n",
       "      <td>play_collect_music</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1001</th>\n",
       "      <td>お気に入りに登録している音楽が聴きたいです</td>\n",
       "      <td>我想听收藏夹的音乐</td>\n",
       "      <td>play_collect_music</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1002 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                        原始文本       中文翻译                  意图  槽值1  槽值2\n",
       "0               エアコンのスイッチONに     打开空调开关             open_ac  NaN  NaN\n",
       "1                    エアコン入れる       打开空调             open_ac  NaN  NaN\n",
       "2                 エアコンのスイッチを     打开空调开关             open_ac  NaN  NaN\n",
       "3               エアコンのスイッチ入れる     打开空调开关             open_ac  NaN  NaN\n",
       "4                エアコンのスイッチON     打开空调开关             open_ac  NaN  NaN\n",
       "...                      ...        ...                 ...  ...  ...\n",
       "997           気に入った楽曲リストをかける  播放喜欢的歌曲列表  play_collect_music  NaN  NaN\n",
       "998            気に入った歌一覧を再生する    播放喜欢的歌曲  play_collect_music  NaN  NaN\n",
       "999   お気に入りのミュージックを聴きたいんですけど   我想听喜欢的音乐  play_collect_music  NaN  NaN\n",
       "1000         音楽のお気に入り一覧を再生する    播放喜欢的音乐  play_collect_music  NaN  NaN\n",
       "1001   お気に入りに登録している音楽が聴きたいです  我想听收藏夹的音乐  play_collect_music  NaN  NaN\n",
       "\n",
       "[1002 rows x 5 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_ja"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "38af1540",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>原始文本</th>\n",
       "      <th>中文翻译</th>\n",
       "      <th>意图</th>\n",
       "      <th>槽值1</th>\n",
       "      <th>槽值2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>open aircon please</td>\n",
       "      <td>请打开空调</td>\n",
       "      <td>open_ac</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>I want to activate the AC</td>\n",
       "      <td>我想打开空调</td>\n",
       "      <td>open_ac</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>I want to turn on the air conditioner</td>\n",
       "      <td>我想打开空调</td>\n",
       "      <td>open_ac</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>switch on the AC please</td>\n",
       "      <td>请打开空调</td>\n",
       "      <td>open_ac</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Help me open the AC</td>\n",
       "      <td>帮我打开空调</td>\n",
       "      <td>open_ac</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>996</th>\n",
       "      <td>I wanna some music from collection list please</td>\n",
       "      <td>我想听收藏的歌曲</td>\n",
       "      <td>play_collect_music</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>997</th>\n",
       "      <td>Listen to favourite music</td>\n",
       "      <td>听收藏的音乐</td>\n",
       "      <td>play_collect_music</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>998</th>\n",
       "      <td>I wanna some songs from favourites</td>\n",
       "      <td>我想听收藏的歌曲</td>\n",
       "      <td>play_collect_music</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>999</th>\n",
       "      <td>Play my favourite list</td>\n",
       "      <td>播放收藏的音乐</td>\n",
       "      <td>play_collect_music</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1000</th>\n",
       "      <td>play favorite song</td>\n",
       "      <td>播放收藏的音乐</td>\n",
       "      <td>play_collect_music</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1001 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                原始文本      中文翻译  \\\n",
       "0                                 open aircon please     请打开空调   \n",
       "1                          I want to activate the AC    我想打开空调   \n",
       "2              I want to turn on the air conditioner    我想打开空调   \n",
       "3                            switch on the AC please     请打开空调   \n",
       "4                                Help me open the AC    帮我打开空调   \n",
       "...                                              ...       ...   \n",
       "996   I wanna some music from collection list please  我想听收藏的歌曲   \n",
       "997                        Listen to favourite music    听收藏的音乐   \n",
       "998               I wanna some songs from favourites  我想听收藏的歌曲   \n",
       "999                           Play my favourite list   播放收藏的音乐   \n",
       "1000                              play favorite song   播放收藏的音乐   \n",
       "\n",
       "                      意图  槽值1  槽值2  \n",
       "0                open_ac  NaN  NaN  \n",
       "1                open_ac  NaN  NaN  \n",
       "2                open_ac  NaN  NaN  \n",
       "3                open_ac  NaN  NaN  \n",
       "4                open_ac  NaN  NaN  \n",
       "...                  ...  ...  ...  \n",
       "996   play_collect_music  NaN  NaN  \n",
       "997   play_collect_music  NaN  NaN  \n",
       "998   play_collect_music  NaN  NaN  \n",
       "999   play_collect_music  NaN  NaN  \n",
       "1000  play_collect_music  NaN  NaN  \n",
       "\n",
       "[1001 rows x 5 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_en"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "08a9c14f",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>原始文本</th>\n",
       "      <th>Unnamed: 1</th>\n",
       "      <th>Unnamed: 2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>エアコンをつける</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>エアコン付ける</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>エアコンをつけてください</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>エアコンをONに</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>エアコン</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>764</th>\n",
       "      <td>お気に入りのミュージックを流して下さい</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>765</th>\n",
       "      <td>気に入った歌リストを再生する</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>766</th>\n",
       "      <td>ミュージックのお気に入り一覧を聴く</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>767</th>\n",
       "      <td>気に入った曲一覧を聴く</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>768</th>\n",
       "      <td>お気に入りのミュージックを流す</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>769 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                    原始文本  Unnamed: 1  Unnamed: 2\n",
       "0               エアコンをつける         NaN         NaN\n",
       "1                エアコン付ける         NaN         NaN\n",
       "2           エアコンをつけてください         NaN         NaN\n",
       "3               エアコンをONに         NaN         NaN\n",
       "4                   エアコン         NaN         NaN\n",
       "..                   ...         ...         ...\n",
       "764  お気に入りのミュージックを流して下さい         NaN         NaN\n",
       "765       気に入った歌リストを再生する         NaN         NaN\n",
       "766    ミュージックのお気に入り一覧を聴く         NaN         NaN\n",
       "767          気に入った曲一覧を聴く         NaN         NaN\n",
       "768      お気に入りのミュージックを流す         NaN         NaN\n",
       "\n",
       "[769 rows x 3 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_ja"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "783b3f70",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "# 任务2：文本分析与文本分词"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "5773e0f0",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Building prefix dict from the default dictionary ...\n",
      "Loading model from cache C:\\Users\\ADMINI~1\\AppData\\Local\\Temp\\jieba.cache\n",
      "Loading model cost 0.867 seconds.\n",
      "Prefix dict has been built successfully.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['汽车', '领域', '多语种', '迁移', '学习', '挑战赛', '初赛', '训练', '集']\n",
      "[pair('汽车', 'n'), pair('领域', 'n'), pair('多语种', 'n'), pair('迁移', 'v'), pair('学习', 'v'), pair('挑战赛', 'n'), pair('初赛', 'n'), pair('训练', 'vn'), pair('集', 'q')]\n"
     ]
    }
   ],
   "source": [
    "import jieba\n",
    "import jieba.posseg as pseg\n",
    "\n",
    "words = jieba.lcut('汽车领域多语种迁移学习挑战赛初赛训练集')\n",
    "print(words)\n",
    "\n",
    "words = pseg.lcut('汽车领域多语种迁移学习挑战赛初赛训练集')\n",
    "print(words)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "e786113f",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[dynet] random seed: 1234\n",
      "[dynet] allocating memory: 32MB\n",
      "[dynet] memory allocation done.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Python/名詞 で/助詞 簡単/形状詞 に/助動詞 使える/動詞 ツール/名詞 です/助動詞\n",
      "['Python', 'で', '簡単', 'に', '使える', 'ツール', 'です']\n"
     ]
    }
   ],
   "source": [
    "import nagisa\n",
    "\n",
    "text = 'Pythonで簡単に使えるツールです'\n",
    "words = nagisa.tagging(text)\n",
    "print(words)\n",
    "# Python/名詞 で/助詞 簡単/形状詞 に/助動詞 使える/動詞 ツール/名詞 です/助動詞\n",
    "\n",
    "print(words.words)\n",
    "# ['Python', 'で', '簡単', 'に', '使える', 'ツール', 'です']"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8ea2bae8",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "# 任务3：TFIDF与文本分类"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "5388eb86",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.pipeline import make_pipeline\n",
    "# 文本分词\n",
    "train_ja['words'] = train_ja['原始文本'].apply(lambda x: ' '.join(nagisa.tagging(x).words))\n",
    "train_en['words'] = train_en['原始文本'].apply(lambda x: x.lower())\n",
    "\n",
    "test_ja['words'] = test_ja['原始文本'].apply(lambda x: ' '.join(nagisa.tagging(x).words))\n",
    "test_en['words'] = test_en['原始文本'].apply(lambda x: x.lower())\n",
    "\n",
    "# 训练TFIDF和逻辑回归\n",
    "pipline = make_pipeline(\n",
    "    TfidfVectorizer(),\n",
    "    LogisticRegression()\n",
    ")\n",
    "pipline.fit(\n",
    "    train_ja['words'].tolist() + train_en['words'].tolist(),\n",
    "    train_ja['意图'].tolist() + train_en['意图'].tolist()\n",
    ")\n",
    "\n",
    "# 模型预测\n",
    "test_ja['意图'] = pipline.predict(test_ja['words'])\n",
    "test_en['意图'] = pipline.predict(test_en['words'])\n",
    "test_en['槽值1'] = np.nan\n",
    "test_en['槽值2'] = np.nan\n",
    "\n",
    "test_ja['槽值1'] = np.nan\n",
    "test_ja['槽值2'] = np.nan\n",
    "\n",
    "# 写入提交文件\n",
    "writer = pd.ExcelWriter('submit.xlsx')\n",
    "test_en.drop(['words'], axis=1).to_excel(writer, sheet_name='英文_testA', index=None)\n",
    "test_ja.drop(['words'], axis=1).to_excel(writer, sheet_name='日语_testA', index=None)\n",
    "writer.save()\n",
    "writer.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a2e00bcb-ec65-4c06-a405-e94b9dd82518",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "# 其余任务"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "200cf583-5d78-4daa-a58c-3e4cd715f6bd",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "见实际py文件"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "775fc616-b7bf-4276-897d-911cc2952364",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}